import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

# Traditional programming vs Machine Learning

print("Traditional Programming:")
print("  Input: Data + Rules")
print("  Output: Answers")
print("  Example: IF price > SMA(20) THEN buy")

print("\nMachine Learning:")
print("  Input: Data + Answers (historical examples)")
print("  Output: Rules (learned patterns)")
print("  Example: Model learns what conditions precede profitable trades")

# Supervised Learning: We have labeled examples
# - Classification: Predict categories (up/down, buy/sell/hold)
# - Regression: Predict continuous values (tomorrow's return)

supervised_examples = {
    'Classification': [
        'Predict if stock goes up or down tomorrow',
        'Classify trades as profitable or not',
        'Detect market regime (bull/bear/sideways)'
    ],
    'Regression': [
        'Predict tomorrow\'s return magnitude',
        'Forecast volatility',
        'Estimate fair value'
    ]
}

# Unsupervised Learning: No labels, find structure in data
unsupervised_examples = {
    'Clustering': [
        'Group similar stocks together',
        'Identify market regimes',
        'Segment trading days by behavior'
    ],
    'Dimensionality Reduction': [
        'Reduce many correlated features to few factors',
        'Find latent market factors',
        'Compress feature space'
    ]
}

print("SUPERVISED LEARNING (with labels):")
for category, examples in supervised_examples.items():
    print(f"\n  {category}:")
    for ex in examples:
        print(f"    - {ex}")

print("\n" + "="*50)
print("\nUNSUPERVISED LEARNING (no labels):")
for category, examples in unsupervised_examples.items():
    print(f"\n  {category}:")
    for ex in examples:
        print(f"    - {ex}")

# The fundamental concept: Split your data

print("Data Split Strategy:")
print("="*50)
print("\n1. TRAINING SET (~60-70%)")
print("   - Model learns patterns from this data")
print("   - Like studying for an exam")

print("\n2. VALIDATION SET (~15-20%)")
print("   - Used to tune hyperparameters")
print("   - Like practice tests")

print("\n3. TEST SET (~15-20%)")
print("   - Final evaluation, NEVER used during training")
print("   - Like the final exam")

# Visual representation
fig, ax = plt.subplots(figsize=(12, 2))

# Draw the splits
ax.barh(0, 0.7, left=0, color='steelblue', label='Training (70%)')
ax.barh(0, 0.15, left=0.7, color='orange', label='Validation (15%)')
ax.barh(0, 0.15, left=0.85, color='green', label='Test (15%)')

ax.set_xlim(0, 1)
ax.set_ylim(-0.5, 0.5)
ax.set_yticks([])
ax.set_xlabel('Data Timeline')
ax.legend(loc='upper center', bbox_to_anchor=(0.5, 1.5), ncol=3)
ax.set_title('Standard Data Split for Time Series')

plt.tight_layout()
plt.show()

# Challenge 1: Low Signal-to-Noise Ratio

import numpy as np
import matplotlib.pyplot as plt

np.random.seed(42)

# Simulate a signal buried in noise
days = 252
signal = np.sin(np.linspace(0, 4*np.pi, days)) * 0.01  # Tiny signal
noise = np.random.normal(0, 0.02, days)  # Much larger noise
observed_returns = signal + noise

# Calculate signal-to-noise ratio
snr = np.std(signal) / np.std(noise)
print(f"Signal-to-Noise Ratio: {snr:.2%}")
print("The true signal is only ~50% as strong as the noise!")

fig, axes = plt.subplots(3, 1, figsize=(12, 8))

axes[0].plot(signal, 'g-', linewidth=2)
axes[0].set_title('True Signal (Hidden)')
axes[0].set_ylabel('Return')

axes[1].plot(noise, 'r-', alpha=0.7)
axes[1].set_title('Noise')
axes[1].set_ylabel('Return')

axes[2].plot(observed_returns, 'b-', alpha=0.7)
axes[2].set_title('What We Observe (Signal + Noise)')
axes[2].set_xlabel('Days')
axes[2].set_ylabel('Return')

plt.tight_layout()
plt.show()

print("\nImplication: Models easily fit to noise, not signal")
print("Solution: Regularization, cross-validation, feature selection")

# Challenge 2: Non-Stationarity

# Financial time series properties change over time
np.random.seed(42)

# Simulate changing volatility regimes
low_vol = np.random.normal(0.001, 0.01, 100)
high_vol = np.random.normal(-0.002, 0.03, 100)
medium_vol = np.random.normal(0.0005, 0.015, 100)

returns = np.concatenate([low_vol, high_vol, medium_vol])

fig, axes = plt.subplots(2, 1, figsize=(12, 6))

# Returns
axes[0].plot(returns)
axes[0].axvline(100, color='r', linestyle='--', alpha=0.7)
axes[0].axvline(200, color='r', linestyle='--', alpha=0.7)
axes[0].set_title('Returns with Changing Regimes')
axes[0].set_ylabel('Return')

# Rolling volatility
rolling_vol = pd.Series(returns).rolling(20).std()
axes[1].plot(rolling_vol, color='orange')
axes[1].axvline(100, color='r', linestyle='--', alpha=0.7)
axes[1].axvline(200, color='r', linestyle='--', alpha=0.7)
axes[1].set_title('Rolling Volatility (20-day)')
axes[1].set_xlabel('Days')
axes[1].set_ylabel('Volatility')

plt.tight_layout()
plt.show()

print("The same strategy that works in one regime may fail in another.")
print("A model trained on low-vol data will struggle in high-vol periods.")

# Challenge 3: Adversarial Environment

print("The Adversarial Nature of Markets")
print("="*50)
print("""
Unlike image classification or language translation:

1. OTHER TRADERS ARE ADAPTING
   - If you find a profitable pattern, others will too
   - As more money exploits a pattern, the edge disappears
   - "Alpha decay" - strategies lose effectiveness over time

2. THE SYSTEM FIGHTS BACK
   - Market makers adjust to trading patterns
   - Large trades move prices against you
   - Information gets priced in faster

3. REGIME CHANGES ARE STRUCTURAL
   - Regulations change market behavior
   - New instruments (ETFs, derivatives) change dynamics
   - Technology (HFT) changes market microstructure

This is fundamentally different from:
- Cats don't evolve to avoid being classified as cats
- Weather patterns don't adapt to forecasts
- Medical diagnoses don't change because you're predicting them
""")

# Challenge 4: Lookahead Bias - The Silent Killer

print("Lookahead Bias Examples")
print("="*50)

lookahead_examples = [
    {
        'mistake': 'Using adjusted close prices for historical signals',
        'problem': 'Adjustments happen after splits/dividends occur',
        'solution': 'Use unadjusted prices, apply adjustments at signal time'
    },
    {
        'mistake': 'Including future data in feature calculation',
        'problem': 'Rolling window includes today\'s close in today\'s signal',
        'solution': 'Use .shift(1) to lag features appropriately'
    },
    {
        'mistake': 'Survivorship bias in stock universe',
        'problem': 'Only including stocks that exist today',
        'solution': 'Use point-in-time constituent lists'
    },
    {
        'mistake': 'Using final earnings numbers',
        'problem': 'Earnings are often revised after initial release',
        'solution': 'Use point-in-time fundamental data'
    }
]

for i, example in enumerate(lookahead_examples, 1):
    print(f"\n{i}. {example['mistake']}")
    print(f"   Problem: {example['problem']}")
    print(f"   Solution: {example['solution']}")

# The Complete ML Pipeline for Trading

class MLPipelineStage:
    """Represents a stage in the ML pipeline."""
    
    def __init__(self, name: str, description: str, key_considerations: list):
        self.name = name
        self.description = description
        self.key_considerations = key_considerations
    
    def display(self):
        print(f"\n{'='*60}")
        print(f"STAGE: {self.name}")
        print(f"{'='*60}")
        print(f"\n{self.description}")
        print("\nKey Considerations:")
        for consideration in self.key_considerations:
            print(f"  - {consideration}")


# Define pipeline stages
pipeline = [
    MLPipelineStage(
        "1. Data Collection",
        "Gather all relevant data sources for your trading strategy.",
        [
            "Price data (OHLCV) at appropriate frequency",
            "Fundamental data (earnings, ratios)",
            "Alternative data (sentiment, satellite)",
            "Ensure point-in-time accuracy"
        ]
    ),
    MLPipelineStage(
        "2. Data Preparation",
        "Clean and prepare data for ML consumption.",
        [
            "Handle missing values appropriately",
            "Adjust for corporate actions",
            "Time series train/test split (no random shuffle!)",
            "Check for lookahead bias"
        ]
    ),
    MLPipelineStage(
        "3. Feature Engineering",
        "Create predictive features from raw data.",
        [
            "Price-based features (returns, volatility)",
            "Technical indicators",
            "Statistical features (z-scores, percentiles)",
            "Feature selection to avoid overfitting"
        ]
    ),
    MLPipelineStage(
        "4. Target Engineering",
        "Define what you're trying to predict.",
        [
            "Return-based vs direction-based targets",
            "Triple barrier method for labeling",
            "Handle overlapping labels",
            "Sample weighting for uniqueness"
        ]
    ),
    MLPipelineStage(
        "5. Model Training",
        "Train and tune your ML model.",
        [
            "Select appropriate algorithm",
            "Hyperparameter tuning with CV",
            "Use time series cross-validation",
            "Ensemble methods for robustness"
        ]
    ),
    MLPipelineStage(
        "6. Evaluation",
        "Assess model performance rigorously.",
        [
            "ML metrics (accuracy, precision, recall)",
            "Financial metrics (Sharpe, returns)",
            "Walk-forward validation",
            "Statistical significance testing"
        ]
    ),
    MLPipelineStage(
        "7. Deployment",
        "Put the model into production.",
        [
            "Real-time feature calculation",
            "Model serving infrastructure",
            "Monitoring and alerting",
            "Retraining schedule"
        ]
    )
]

# Display all stages
for stage in pipeline:
    stage.display()

# Mini Example: Complete Pipeline Demo

import pandas as pd
import numpy as np
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import accuracy_score
import yfinance as yf

print("Mini Pipeline Demo")
print("="*50)

# Stage 1: Data Collection
print("\n1. Collecting data...")
df = yf.Ticker("AAPL").history(period="2y")
print(f"   Downloaded {len(df)} rows")

# Stage 2: Data Preparation
print("\n2. Preparing data...")
df = df[['Open', 'High', 'Low', 'Close', 'Volume']].copy()
print(f"   Columns: {list(df.columns)}")

# Stage 3: Feature Engineering
print("\n3. Engineering features...")
df['returns'] = df['Close'].pct_change()
df['volatility'] = df['returns'].rolling(20).std()
df['momentum'] = df['Close'].pct_change(10)
df['volume_change'] = df['Volume'].pct_change()
print(f"   Created 4 features")

# Stage 4: Target Engineering
print("\n4. Creating target...")
df['target'] = (df['returns'].shift(-1) > 0).astype(int)
print(f"   Target: Next day direction (0=down, 1=up)")

# Prepare final dataset
features = ['returns', 'volatility', 'momentum', 'volume_change']
df_clean = df.dropna()

# Stage 5: Model Training (with time series split)
print("\n5. Training model...")
split_idx = int(len(df_clean) * 0.8)

X_train = df_clean[features][:split_idx]
y_train = df_clean['target'][:split_idx]
X_test = df_clean[features][split_idx:]
y_test = df_clean['target'][split_idx:]

model = RandomForestClassifier(n_estimators=100, max_depth=5, random_state=42)
model.fit(X_train, y_train)
print(f"   Trained on {len(X_train)} samples")

# Stage 6: Evaluation
print("\n6. Evaluating model...")
y_pred = model.predict(X_test)
accuracy = accuracy_score(y_test, y_pred)
print(f"   Test accuracy: {accuracy:.2%}")
print(f"   Baseline (random): 50%")

# Stage 7: Deployment (just a preview)
print("\n7. Ready for deployment!")
print(f"   Model can predict direction based on 4 features")

# Core ML Libraries

print("Essential Libraries for Financial ML")
print("="*50)

libraries = {
    'Data Manipulation': {
        'pandas': 'DataFrames for tabular data',
        'numpy': 'Numerical computing',
    },
    'Machine Learning': {
        'scikit-learn': 'Core ML algorithms and utilities',
        'xgboost': 'Gradient boosting (fast, accurate)',
        'lightgbm': 'Another gradient boosting option',
    },
    'Deep Learning': {
        'torch (PyTorch)': 'Neural networks',
        'tensorflow': 'Alternative to PyTorch',
    },
    'Visualization': {
        'matplotlib': 'Static plots',
        'seaborn': 'Statistical visualizations',
        'plotly': 'Interactive plots',
    },
    'Financial Data': {
        'yfinance': 'Yahoo Finance data',
        'pandas-datareader': 'Multiple data sources',
    },
    'NLP & Sentiment': {
        'nltk': 'Natural language processing',
        'transformers': 'Pre-trained language models',
    },
    'Model Interpretation': {
        'shap': 'SHAP values for explainability',
        'lime': 'Local interpretable explanations',
    }
}

for category, libs in libraries.items():
    print(f"\n{category}:")
    for lib, desc in libs.items():
        print(f"  {lib}: {desc}")

# Check your environment

def check_library(name: str) -> tuple:
    """Check if a library is installed and get its version."""
    try:
        module = __import__(name.replace('-', '_'))
        version = getattr(module, '__version__', 'unknown')
        return True, version
    except ImportError:
        return False, None

print("Environment Check")
print("="*50)

essential_libs = ['pandas', 'numpy', 'sklearn', 'matplotlib', 'yfinance']
optional_libs = ['xgboost', 'lightgbm', 'torch', 'shap']

print("\nEssential Libraries:")
for lib in essential_libs:
    installed, version = check_library(lib)
    status = f"v{version}" if installed else "NOT INSTALLED"
    symbol = "OK" if installed else "MISSING"
    print(f"  [{symbol}] {lib}: {status}")

print("\nOptional Libraries:")
for lib in optional_libs:
    installed, version = check_library(lib)
    status = f"v{version}" if installed else "not installed"
    symbol = "OK" if installed else "--"
    print(f"  [{symbol}] {lib}: {status}")

# Jupyter Notebook Best Practices for ML

print("Best Practices for ML in Jupyter")
print("="*50)

best_practices = [
    ("Set random seeds", "np.random.seed(42) for reproducibility"),
    ("Suppress warnings carefully", "warnings.filterwarnings('ignore') when appropriate"),
    ("Use autoreload", "%load_ext autoreload for module development"),
    ("Track experiments", "Log hyperparameters and results systematically"),
    ("Checkpoint models", "Save model state periodically"),
    ("Memory management", "Delete large objects with del, use gc.collect()"),
    ("Version control", "Strip outputs before committing notebooks")
]

for practice, explanation in best_practices:
    print(f"\n{practice}:")
    print(f"  {explanation}")

# Module Project: ML Project Template

import os
from datetime import datetime
from typing import Dict, List, Optional
import json


class MLProjectTemplate:
    """
    Creates a standardized project structure for financial ML projects.
    
    This template ensures consistent organization and includes
    all necessary components for reproducible ML experiments.
    """
    
    def __init__(self, project_name: str, description: str = ""):
        """
        Initialize a new ML project template.
        
        Args:
            project_name: Name of the project
            description: Brief description of the project
        """
        self.project_name = project_name
        self.description = description
        self.created_at = datetime.now().isoformat()
        self.config = self._default_config()
        self.directory_structure = self._default_structure()
    
    def _default_config(self) -> Dict:
        """Return default project configuration."""
        return {
            'data': {
                'source': 'yfinance',
                'symbols': [],
                'period': '2y',
                'test_size': 0.2,
                'validation_size': 0.1
            },
            'features': {
                'price_based': ['returns', 'volatility', 'momentum'],
                'technical': ['sma', 'rsi', 'macd'],
                'scaling': 'standard'
            },
            'target': {
                'type': 'classification',
                'method': 'direction',
                'horizon': 1
            },
            'model': {
                'algorithm': 'random_forest',
                'cv_method': 'time_series',
                'n_splits': 5,
                'hyperparameters': {}
            },
            'evaluation': {
                'ml_metrics': ['accuracy', 'precision', 'recall', 'f1'],
                'financial_metrics': ['sharpe', 'returns', 'max_drawdown']
            },
            'random_seed': 42
        }
    
    def _default_structure(self) -> Dict:
        """Return default directory structure."""
        return {
            'data': {
                'raw': 'Original data files',
                'processed': 'Cleaned and transformed data',
                'features': 'Engineered features'
            },
            'notebooks': {
                '01_data_exploration.ipynb': 'EDA notebook',
                '02_feature_engineering.ipynb': 'Feature creation',
                '03_model_training.ipynb': 'Model development',
                '04_evaluation.ipynb': 'Results analysis'
            },
            'src': {
                'data': 'Data loading and processing modules',
                'features': 'Feature engineering code',
                'models': 'Model definitions',
                'evaluation': 'Evaluation utilities'
            },
            'models': 'Saved model files',
            'reports': 'Generated reports and visualizations',
            'config': 'Configuration files'
        }
    
    def set_symbols(self, symbols: List[str]) -> 'MLProjectTemplate':
        """Set the trading symbols for the project."""
        self.config['data']['symbols'] = symbols
        return self
    
    def set_model(self, algorithm: str, **kwargs) -> 'MLProjectTemplate':
        """Set the model algorithm and hyperparameters."""
        self.config['model']['algorithm'] = algorithm
        self.config['model']['hyperparameters'] = kwargs
        return self
    
    def set_target(self, target_type: str, method: str, horizon: int = 1) -> 'MLProjectTemplate':
        """Set the prediction target configuration."""
        self.config['target'] = {
            'type': target_type,
            'method': method,
            'horizon': horizon
        }
        return self
    
    def validate_config(self) -> Dict:
        """
        Validate the project configuration.
        
        Returns:
            Dictionary with validation results and warnings
        """
        results = {
            'valid': True,
            'errors': [],
            'warnings': []
        }
        
        # Check required fields
        if not self.config['data']['symbols']:
            results['warnings'].append("No symbols specified")
        
        # Check for common mistakes
        test_size = self.config['data']['test_size']
        val_size = self.config['data']['validation_size']
        if test_size + val_size > 0.5:
            results['warnings'].append(
                f"Test + validation = {test_size + val_size:.0%}, leaving only "
                f"{1 - test_size - val_size:.0%} for training"
            )
        
        # Check model configuration
        if self.config['model']['cv_method'] == 'random':
            results['errors'].append(
                "Random CV is inappropriate for time series data"
            )
            results['valid'] = False
        
        return results
    
    def generate_readme(self) -> str:
        """Generate a README for the project."""
        readme = f"""
# {self.project_name}

{self.description}

Created: {self.created_at}

## Configuration

### Data
- Source: {self.config['data']['source']}
- Symbols: {', '.join(self.config['data']['symbols']) or 'Not specified'}
- Period: {self.config['data']['period']}

### Model
- Algorithm: {self.config['model']['algorithm']}
- CV Method: {self.config['model']['cv_method']}

### Target
- Type: {self.config['target']['type']}
- Method: {self.config['target']['method']}
- Horizon: {self.config['target']['horizon']} day(s)

## Directory Structure

```
{self.project_name}/
├── data/
│   ├── raw/
│   ├── processed/
│   └── features/
├── notebooks/
├── src/
│   ├── data/
│   ├── features/
│   ├── models/
│   └── evaluation/
├── models/
├── reports/
└── config/
```

## Usage

1. Start with `notebooks/01_data_exploration.ipynb`
2. Engineer features in `notebooks/02_feature_engineering.ipynb`
3. Train models in `notebooks/03_model_training.ipynb`
4. Evaluate results in `notebooks/04_evaluation.ipynb`
"""
        return readme.strip()
    
    def get_summary(self) -> str:
        """Get a summary of the project template."""
        validation = self.validate_config()
        
        summary = f"""
{'='*60}
ML PROJECT TEMPLATE: {self.project_name}
{'='*60}

Description: {self.description or 'Not provided'}
Created: {self.created_at}

CONFIGURATION:
  Data Source: {self.config['data']['source']}
  Symbols: {self.config['data']['symbols'] or 'Not set'}
  Model: {self.config['model']['algorithm']}
  Target: {self.config['target']['method']} ({self.config['target']['type']})

VALIDATION: {'PASSED' if validation['valid'] else 'FAILED'}
"""
        
        if validation['errors']:
            summary += "\nERRORS:\n"
            for error in validation['errors']:
                summary += f"  - {error}\n"
        
        if validation['warnings']:
            summary += "\nWARNINGS:\n"
            for warning in validation['warnings']:
                summary += f"  - {warning}\n"
        
        return summary


# Demo the project template
print("Creating ML Project Template...\n")

project = MLProjectTemplate(
    project_name="SPY Direction Predictor",
    description="Predict next-day direction of SPY using technical features"
)

# Configure the project
project.set_symbols(['SPY'])
project.set_model(
    'random_forest',
    n_estimators=100,
    max_depth=5
)
project.set_target('classification', 'direction', horizon=1)

# Display summary
print(project.get_summary())

# Show README preview
print("\n" + "="*60)
print("README PREVIEW:")
print("="*60)
print(project.generate_readme()[:1000] + "...")

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import yfinance as yf
import warnings
warnings.filterwarnings('ignore')

# Download sample data
print("Downloading sample data...")
df = yf.Ticker("AAPL").history(period="2y")
print(f"Downloaded {len(df)} rows")
print(f"\nColumns: {list(df.columns)}")
print(f"\nDate range: {df.index[0].date()} to {df.index[-1].date()}")

# Check for missing data

def analyze_missing_data(df: pd.DataFrame) -> pd.DataFrame:
    """
    Analyze missing data patterns in a DataFrame.
    
    Args:
        df: Input DataFrame
        
    Returns:
        DataFrame with missing data statistics
    """
    missing_stats = pd.DataFrame({
        'missing_count': df.isnull().sum(),
        'missing_pct': df.isnull().sum() / len(df) * 100,
        'dtype': df.dtypes
    })
    return missing_stats.sort_values('missing_pct', ascending=False)

print("Missing Data Analysis:")
print(analyze_missing_data(df))

# Handling missing values in financial data

class FinancialDataCleaner:
    """
    Clean financial time series data with appropriate methods.
    """
    
    def __init__(self, df: pd.DataFrame):
        self.df = df.copy()
        self.cleaning_log = []
    
    def handle_missing_prices(self, method: str = 'ffill') -> 'FinancialDataCleaner':
        """
        Handle missing price data.
        
        For OHLC data, forward fill is usually appropriate as it
        represents "last known price" - what you'd actually trade at.
        """
        price_cols = ['Open', 'High', 'Low', 'Close']
        price_cols = [c for c in price_cols if c in self.df.columns]
        
        before_missing = self.df[price_cols].isnull().sum().sum()
        
        if method == 'ffill':
            self.df[price_cols] = self.df[price_cols].ffill()
        elif method == 'interpolate':
            self.df[price_cols] = self.df[price_cols].interpolate(method='time')
        
        after_missing = self.df[price_cols].isnull().sum().sum()
        self.cleaning_log.append(
            f"Filled {before_missing - after_missing} missing price values using {method}"
        )
        
        return self
    
    def handle_missing_volume(self, fill_value: float = 0) -> 'FinancialDataCleaner':
        """
        Handle missing volume data.
        
        Missing volume often means no trades occurred, so filling with 0 is reasonable.
        """
        if 'Volume' in self.df.columns:
            before_missing = self.df['Volume'].isnull().sum()
            self.df['Volume'] = self.df['Volume'].fillna(fill_value)
            self.cleaning_log.append(
                f"Filled {before_missing} missing volume values with {fill_value}"
            )
        return self
    
    def detect_outliers(self, column: str, n_std: float = 5) -> pd.Series:
        """
        Detect outliers using standard deviation method.
        
        For financial data, 5 std is a reasonable threshold as
        returns can legitimately be extreme.
        """
        if column not in self.df.columns:
            return pd.Series(dtype=bool)
        
        mean = self.df[column].mean()
        std = self.df[column].std()
        
        lower_bound = mean - n_std * std
        upper_bound = mean + n_std * std
        
        is_outlier = (self.df[column] < lower_bound) | (self.df[column] > upper_bound)
        
        self.cleaning_log.append(
            f"Found {is_outlier.sum()} outliers in {column} (>{n_std} std)"
        )
        
        return is_outlier
    
    def remove_zero_volume_days(self) -> 'FinancialDataCleaner':
        """
        Remove days with zero volume (likely data errors or holidays).
        """
        if 'Volume' in self.df.columns:
            before_len = len(self.df)
            self.df = self.df[self.df['Volume'] > 0]
            removed = before_len - len(self.df)
            self.cleaning_log.append(f"Removed {removed} zero-volume days")
        return self
    
    def get_cleaned_data(self) -> pd.DataFrame:
        """Return the cleaned DataFrame."""
        return self.df
    
    def get_cleaning_report(self) -> str:
        """Return a report of all cleaning operations."""
        report = "Data Cleaning Report\n" + "="*40 + "\n"
        for entry in self.cleaning_log:
            report += f"- {entry}\n"
        return report


# Apply cleaning
cleaner = FinancialDataCleaner(df)
cleaner.handle_missing_prices('ffill')
cleaner.handle_missing_volume(0)
cleaner.remove_zero_volume_days()

# Add returns and check for outliers
df_clean = cleaner.get_cleaned_data()
df_clean['returns'] = df_clean['Close'].pct_change()
cleaner.df = df_clean
outliers = cleaner.detect_outliers('returns', n_std=5)

print(cleaner.get_cleaning_report())
print(f"\nFinal dataset: {len(df_clean)} rows")

# Why random split fails for time series

print("Random Split vs Time Series Split")
print("="*50)

print("""
RANDOM SPLIT (WRONG for time series):
┌─────────────────────────────────────────┐
│ Train Train Test Train Test Train Test  │
│   ↑      ↑     ↑     ↑     ↑     ↑   ↑  │
│   Randomly scattered across time        │
└─────────────────────────────────────────┘
Problem: Model can "peek" at future data during training!

TIME SERIES SPLIT (CORRECT):
┌─────────────────────────────────────────┐
│ Train Train Train Train │ Test Test Test│
│ ←─── Earlier ──→        │ ←─ Later ──→  │
└─────────────────────────────────────────┘
Correct: Model only trained on past, tested on future.
""")

# Proper time series split implementation

from typing import Tuple

def time_series_split(
    df: pd.DataFrame,
    test_size: float = 0.2,
    validation_size: float = 0.1
) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
    """
    Split time series data maintaining temporal order.
    
    Args:
        df: DataFrame sorted by date
        test_size: Proportion for test set (most recent data)
        validation_size: Proportion for validation set
        
    Returns:
        Tuple of (train, validation, test) DataFrames
    """
    n = len(df)
    
    # Calculate split indices
    test_start = int(n * (1 - test_size))
    val_start = int(n * (1 - test_size - validation_size))
    
    # Split
    train = df.iloc[:val_start]
    validation = df.iloc[val_start:test_start]
    test = df.iloc[test_start:]
    
    return train, validation, test


# Apply split
train, val, test = time_series_split(df_clean, test_size=0.2, validation_size=0.1)

print("Time Series Split Results:")
print("="*50)
print(f"\nTraining:   {len(train):4d} rows ({len(train)/len(df_clean)*100:.1f}%)")
print(f"  Period: {train.index[0].date()} to {train.index[-1].date()}")
print(f"\nValidation: {len(val):4d} rows ({len(val)/len(df_clean)*100:.1f}%)")
print(f"  Period: {val.index[0].date()} to {val.index[-1].date()}")
print(f"\nTest:       {len(test):4d} rows ({len(test)/len(df_clean)*100:.1f}%)")
print(f"  Period: {test.index[0].date()} to {test.index[-1].date()}")

# Visualize the split

fig, ax = plt.subplots(figsize=(14, 5))

ax.plot(train.index, train['Close'], 'b-', label='Training', linewidth=1)
ax.plot(val.index, val['Close'], 'orange', label='Validation', linewidth=1)
ax.plot(test.index, test['Close'], 'g-', label='Test', linewidth=1)

# Add vertical lines at split points
ax.axvline(val.index[0], color='gray', linestyle='--', alpha=0.7)
ax.axvline(test.index[0], color='gray', linestyle='--', alpha=0.7)

ax.set_title('Time Series Train/Validation/Test Split')
ax.set_xlabel('Date')
ax.set_ylabel('Price')
ax.legend()
ax.grid(True, alpha=0.3)

plt.tight_layout()
plt.show()